import requests
import threading
from bs4 import BeautifulSoup

urls = [f"https://www.baidu.com/#p{page}" for page in range(1, 50+1)]

def craw(url):
    r = requests.get(url)
    return r.text

def parse(html):
    soup = BeautifulSoup(html, 'html.parser')
    links = soup.find_all("a", class_="post-item-title")  # html 中 a 标签，class_后面是 a 标签内容
    return [(link["href"], link.get_text()) for link in links]


# ---------------------------------------
import concurrent.futures

# craw
with concurrent.futures.ThreadPoolExecutor() as pool:
    htmls = pool.map(craw, urls)
    htmls = list(zip(urls, htmls))
    for url, html in htmls:
        print(url, len(html))

print('craw over')

# parse
with concurrent.futures.ThreadPoolExecutor() as pool:
    futures = {}
    for url, html in htmls:
        future = pool.submit(parse, html)
        futures[future] = url

    # 方式1
    # for future, url in futures.items():
    #     print(url, future.result())

    # 方式2  (无序, 先完成先返回)
    for future in concurrent.futures.as_completed(futures):
        url = futures[future]
        print(url, future.result())


